# -*- coding: utf-8 -*-

BOT_NAME = 'douban_group_spider'
SPIDER_MODULES = ['douban_group_spider.spiders']
NEWSPIDER_MODULE = 'douban_group_spider.spiders'

#使用scrapy-redis内置的去重组件
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
#使用scrapy-redis调度器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 使用队列形式
SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderQueue"
#允许暂停，redis请求记录不丢失
SCHEDULER_PERSIST = True

ROBOTSTXT_OBEY = False #不遵守robot协议

DOWNLOAD_DELAY = 1 #间隔时间


COOKIES_ENABLED = False

DOWNLOADER_MIDDLEWARES = {
    'douban_group_spider.middlewares.DoubanGroupSpiderSpiderMiddleware': 543,
    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 560,
    'douban_group_spider.middlewares.ABProxyMiddleware' : 550,
    'douban_group_spider.middlewares.UserAgentMiddleware': 600,
}

ITEM_PIPELINES = {
    'douban_group_spider.pipelines.DoubanGroupSpiderPipeline': 300,
    'scrapy_redis.pipelines.RedisPipeline' : 900, #支持将数据存储到redis中，必须启动
}


# 阿布云ip代理配置，包括账号密码
import base64
PROXY_USER = "HL0S22561DO625TD"
PROXY_PASS = "9127D4370BEE3D4B"
PROXY_HOST = "http-dyn.abuyun.com"
PROXY_PORT = "9020"
# for Python3
PROXY_AUTH = "Basic " + base64.urlsafe_b64encode(bytes((PROXY_USER + ":" + PROXY_PASS), "ascii")).decode("utf8")
PROXY_SERVER = "http://" + PROXY_HOST +  ":" + PROXY_PORT


#配置redis数据库
REDIS_HOST = '192.168.1.130'
REDIS_PORT = 9901
REDIS_DB = 0
REDIS_URL = 'redis://' + REDIS_HOST + ': '+ str(REDIS_PORT) + "/" + str(REDIS_DB)
